package com.chance.cc.crawler.development.bootstrap.pdd;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.downloader.HttpConfig;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.downloader.HttpRequest;
import com.chance.cc.crawler.core.downloader.HttpRequestBody;
import com.chance.cc.crawler.core.downloader.http.HttpClientDownloader;
import com.chance.cc.crawler.core.downloader.http.HttpClientGenerator;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.downloader.proxy.SiteConfigProxyProvider;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.development.utils.RedisReader;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.net.URLEncoder;
import java.util.*;

import static com.chance.cc.crawler.core.script.crawler.CrawlerScript.$_type;
import static com.chance.cc.crawler.development.bootstrap.douyin.DYTest.genKeywords;
import static com.chance.cc.crawler.development.bootstrap.douyin.DYTest.genShops;

/**
 * @author bx
 * @date 2021/4/27 0027 12:51
 */
public class PDDTest {

    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");
    }

    private static Proxy cluProxy = new Proxy();
    static {
        cluProxy.setHost("http-cla.abuyun.com");
        cluProxy.setPort(9030);
        cluProxy.setUsername("H3487178Q0I1HVPC");
        cluProxy.setPassword("ACE81171D81169CA");
    }

    public static Object ipLock = new Object();


    public static Set<String> ips = new HashSet<>();

    public static String holdIP = null;

    public static Logger logger = LoggerFactory.getLogger(PDDTest.class);

    public static void main(String[] args) throws IOException, InterruptedException {
        HttpConfig httpConfig = HttpConfig.me("test1");
        httpConfig.setProxy(proxy);
        httpConfig.setUseProxy(true);

        HttpClientDownloader downloader = new HttpClientDownloader();
        downloader.setClientGenerator(new HttpClientGenerator());
        downloader.setProxyProvider(new SiteConfigProxyProvider());

        for (int i = 1; i <= 1; i++) {
            new Thread(new TestShopHtmlRun(downloader, httpConfig), "thread_" + i).start();
        }
//        statics();
//        statics1();

    }

    private static void statics1() throws IOException {
        List<String> lines26 = IOUtils.readLines(new FileInputStream("D:\\chance\\data\\pdd\\pdd_1.json"),"utf-8");
        Set<String> goodsIds = new HashSet<>();
        for (String s : lines26) {
            String goodsId = new Json(s).jsonPath("$.goods_id").get();
            goodsIds.add(goodsId);
        }
        System.out.println("类别商品信息数量：" + goodsIds.size());
        
        RedisReader redisReader = new RedisReader("192.168.1.215", 6379, 2);
        Set<String> smembers = redisReader.smembers("pdd-goods-id-queue_set");
        System.out.println("搜索商品信息数量：" + smembers.size());
        int count =0;
        for (String smember : smembers) {
            if (!goodsIds.contains(smember)){
                System.out.println("类别中不包含的商品："+ smember);
                count++;
            }
        }
        System.out.println("类别中不包含的商品数量："+ count);
    }

//

    static class TestSearchRun implements Runnable {

        HttpClientDownloader downloader;
        HttpConfig httpConfig;

        public TestSearchRun(HttpClientDownloader downloader, HttpConfig httpConfig) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
        }


        @Override
        public void run() {
            String firstUrlFormat = "https://api.pinduoduo.com/search?install_token=d0c17b96-9658-4503-a19c-1756d19f70c3&item_ver=lzqq&list_id=qyuMiT4a&search_met=history&sort=default&source=index" +
                    "&q=%s" +
                    "&page_sn=10015&page_id=search_result.html&size=20&requery=0" +
                    "&page=%s" +
                    "&engine_version=2.0&is_new_query=1&back_search=false&pdduid=1720023679323";
            String otherUlrFormat = "https://api.pinduoduo.com/search?install_token=d0c17b96-9658-4503-a19c-1756d19f70c3&item_ver=lzqq&list_id=qyuMiT4a&search_met=history&sort=default&source=index" +
                    "&q=%s" +
                    "&page_sn=10015&page_id=search_result.html&size=20&requery=0" +
                    "&page=%s" +
                    "&flip=%s" +
                    "&engine_version=2.0&is_new_query=1&back_search=false&pdduid=1720023679323";
            Map<String,String> headers = new HashMap<>();
            headers.put("X-PDD-QUERIES","width=1080&height=1920&net=1&brand=Xiaomi&model=MI+5s&osv=8.0.0&appv=5.59.0&pl=2");
            headers.put("ETag","wYmUAFKX");
            headers.put("AccessToken","CACMOJMI4VRPEIJOMGB3CWTMRM7VT4ITQZFJPF75RAWIA2WWB7DA1115bc2");
            headers.put("Referer","Android");
            headers.put("Content-Type","application/json;charset=UTF-8");
            headers.put("p-mediainfo","player=2.0.3&rtc=3.0.0");
            headers.put("p-appname","pinduoduo");
            headers.put("x-pdd-info","bold_free%3Dfalse%26bold_product%3D");
            headers.put("p-proc","main");
            headers.put("User-Agent","android Mozilla/5.0 (Linux; Android 8.0.0; MI 5s Build/OPR1.170623.032; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.99 Mobile Safari/537.36 phh_android_version/5.59.0 phh_android_build/8580855ea13a70c6bd60b68f6214008b255cc9af phh_android_channel/yyb pversion/0");
            headers.put("PDD-CONFIG","V4:001.055900");
            headers.put("multi-set","0,0");
            headers.put("Host","api.pinduoduo.com");
            headers.put("Connection","Keep-Alive");
            headers.put("Accept-Encoding","gzip");
            List<String> keyWords = genKeywords();
            HttpRequest httpRequest = new HttpRequest();
            httpRequest.setHeaders(headers);
            HttpPage httpPage;
            for (String keyWord : keyWords) {
                try {
                    String url = String.format(firstUrlFormat, URLEncoder.encode(keyWord,"utf-8"),1);
                    httpRequest.setUrl(url);
                    httpPage = downloader.download(httpRequest, httpConfig);
                    Json json = new Json(httpPage.getRawText());
                    int page = 1;

                    while (page < 100){
                        page++;
                        List<String> all = json.jsonPath($_type + ".items").all();
                        for (String s : all) {
                            System.out.println(s);
                        }

                        String flip = json.jsonPath($_type + ".flip").get();

                        url = String.format(otherUlrFormat, URLEncoder.encode(keyWord,"utf-8"),page,flip);
                        httpRequest.setUrl(url);
                        httpPage = downloader.download(httpRequest, httpConfig);
                        json = new Json(httpPage.getRawText());

                    }
                } catch (Exception e) {
                    e.printStackTrace();
                }


            }

        }
    }

    static class TestShopRun implements Runnable {

        HttpClientDownloader downloader;
        HttpConfig httpConfig;
        private static final String urlFormat = "https://api.pinduoduo.com/api/turing/mall/query_cat_goods?sort_type=default" +
                "&category_id=%s" +
                "&list_id=4c4f8c834d" +
                "&mall_id=%s" +
                "&refer_page_sn=10015" +
                "&page_no=%s" +
                "&type=0&page_size=20&pdduid=3848719808912";

        public TestShopRun(HttpClientDownloader downloader, HttpConfig httpConfig) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
        }


        @Override
        public void run() {


            Map<String, String> headers = new HashMap<>();
            headers.put("X-PDD-QUERIES", "width=1080&height=1920&net=1&brand=Xiaomi&model=MI+5s&osv=8.0.0&appv=5.59.0&pl=2");
            headers.put("ETag", "jVYb8J44");
            headers.put("AccessToken", "36W7PJGKRND6DOLFHUG6CJWUSKMPMMK3WUCNVJD7HQCGWPJL4GKA111903b");
            headers.put("Referer", "Android");
            headers.put("Content-Type", "application/json;charset=UTF-8");
            headers.put("p-mediainfo", "player=2.0.3&rtc=3.0.0");
            headers.put("p-appname", "pinduoduo");
            headers.put("x-pdd-info", "bold_free%3Dfalse%26bold_product%3D");
            headers.put("p-proc", "main");
            headers.put("User-Agent", "android Mozilla/5.0 (Linux; Android 8.0.0; MI 5s Build/OPR1.170623.032; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.99 Mobile Safari/537.36 phh_android_version/5.59.0 phh_android_build/8580855ea13a70c6bd60b68f6214008b255cc9af phh_android_channel/yyb pversion/0");
            headers.put("PDD-CONFIG", "V4:001.055900");
            headers.put("multi-set", "0,0");
            headers.put("Host", "api.pinduoduo.com");
            headers.put("Connection", "Keep-Alive");
            headers.put("Accept-Encoding", "gzip");
            headers.put("anti-token", "2afLtfp6akMe49GAifkJn7oPo4Loo/o7f+2u/8V9CBxNjoYMHOFBilYChB9jAJ/WLZDxDsUBPRa/Gjnw+vZ9INR5V0m4fisYDHn7zRv4oHHsNaIPYV7wfu/OpO1v3BRCcDrfYGX1xZ241Zu4LcrMONfkjVoQm2R3T7f34Lh+xpSA6McA7pkk2u0jMYe3NKwiK582cATTFH0Ukha0DsKcLRmeoytnogGtZg1NO5IWQPQWTcvJaSaUOXmyLmsW7xlEW3VSvFP9gsidMzk+LHeuyWE8D9Sa57cERUQgq8aZbryPLYqgvHP/62GcKP6V/FF6/PJBJGzZpj42Cp7Fky1Z4bJRixCZTIScyy8jPF46ovwNMZGj+x6DNk5mvpmAg+BN4+XW0IvJg40Ew+rUH1SPx6n1m4/Zz41cd+rnkM9rHuOfbe5p1kbI5VrrwJkewUDrlj0WbhTFSE+nCHxUGzBrLZM4cbBogvnxQvPhjDkIPHwQhOuZ45TnYIu5f7yuEo7ctytGSOfOcCHwZfRdYMrGpspcqIzs1K+DeaCqYQh9aJKEBg=");
            List<String> shops = genShops();
            FileOutputStream fileOutputStream = null;
            try {
                fileOutputStream = new FileOutputStream("D:\\chance\\data\\pdd\\pdd_shop.json", true);
            } catch (FileNotFoundException e) {
                e.printStackTrace();
            }
            RedisReader redisReader = new RedisReader("192.168.1.215", 6379, 2);
            //category_id=23082 婴儿奶粉
            //category_id=23994 儿童奶粉
            for (String shop : shops) {
                try {
                    downloadBabyMilk("23082",shop,downloader,httpConfig,headers,fileOutputStream,redisReader);
                    downloadBabyMilk("23994",shop,downloader,httpConfig,headers,fileOutputStream,redisReader);
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        }

        private void downloadBabyMilk(String category_id,String shop,
                                      HttpClientDownloader downloader, HttpConfig httpConfig,
                                      Map<String, String> headers,FileOutputStream fileOutputStream,
                                      RedisReader redisReader) throws InterruptedException, IOException {
            String url = String.format(urlFormat, category_id, shop, 1);
            String goodIdsQueue = "pdd-shop-goods-id-queue_set";
            HttpRequest httpRequest = new HttpRequest();
            httpRequest.setHeaders(headers);
            HttpPage httpPage;
            httpRequest.setUrl(url);
            httpPage = downloader.download(httpRequest, httpConfig);
            Json json = new Json(httpPage.getRawText());

            while (true) {
                Random rand = new Random();
                long sleepTime = (long)((rand.nextFloat()+1.5)*2000L);
                Thread.sleep(sleepTime);

                List<String> all = json.jsonPath($_type + ".goods_list").all();
                if (all == null ||all.size() < 1){
                    break;
                }
                for (String s : all) {
                    try {
                        Json goodJson = new Json(s);
                        String goods_id = goodJson.jsonPath("$.goods_id").get();
                        if (redisReader.sismember(goodIdsQueue,goods_id)){
                            System.out.println("重复id："+goods_id);
                        }
                        redisReader.sadd(goodIdsQueue,goods_id);
                    } catch (Exception e) {
                        logger.error("wash detail {}",s);
                    }
                }

                IOUtils.writeLines(all,"\r\n",fileOutputStream,"utf-8");

                int page_no = Integer.valueOf(json.jsonPath($_type + ".page_no").get());

                url = String.format(urlFormat, category_id, shop, ++page_no);
                httpRequest.setUrl(url);
                httpPage = downloader.download(httpRequest, httpConfig);
                json = new Json(httpPage.getRawText());

            }
        }
    }

    static class TestShopHtmlRun implements Runnable {

        List<String> accessTokens = new ArrayList<>();
        Map<String,String> accessTokenMap = new HashMap<>();
        HttpClientDownloader downloader;
        HttpConfig httpConfig;
        private static final String urlFormat = "http://yangkeduo.com/proxy/api/api/turing/mall/query_cat_goods?" +
                "pdduid=6968271439966&type=0" +
                "&category_id=%s" +
                "&mall_id=%s" +
                "&page_no=%s" +
                "&page_size=20&sort_type=default";

        public TestShopHtmlRun(HttpClientDownloader downloader, HttpConfig httpConfig) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;

//            accessTokens.add("AVOM7SFKWMWC2JOQOKLAUD4F3EKZV6D56ZQGK5BX2V3WI7FC4R2A1106857"); //18019432723
            accessTokens.add("VFN4J2BF4BVDFTIWN7O6LYU5O4FMRJWRG2DYCQTLMEEGQXIBOVHQ1105ea2"); //17721250791
//            accessTokens.add("H2ZYFKK3MLASUJJ4WP6FNUBTUDRQCPAWBDGSAZBXXNI5KIVLASUA112d599"); //17765187595



            accessTokenMap.put("H2ZYFKK3MLASUJJ4WP6FNUBTUDRQCPAWBDGSAZBXXNI5KIVLASUA112d599","huxRiclZtMS9tJH3d_4RtA987868d801c479b30");
            accessTokenMap.put("D4IKSPFNHYSCWJ4TDIBTJPZJS2OSGLER7HCUMJTHG7D4UHCLPQVQ1105ea2","_PbZdfIXh4A_URwa_TYMlw2a4e67ca2fca4ce5e");
        }


        @Override
        public void run() {

            String accessToken = accessTokens.get(0);
            Map<String, String> headers = new HashMap<>();
            headers.put("Accept", "application/json, text/plain, */*");
            headers.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36");
            List<String> shops = genShops();
            FileOutputStream fileOutputStream = null;
            try {
                fileOutputStream = new FileOutputStream("D:\\chance\\data\\pdd\\pdd_html_shop.json", true);
            } catch (FileNotFoundException e) {
                e.printStackTrace();
            }
            RedisReader redisReader = new RedisReader("192.168.1.215", 6379, 2);
            //category_id=23082 婴儿奶粉
            //category_id=23994 儿童奶粉
            //category_id=23017 奶粉营养
            while (true) {
                headers.put("AccessToken", accessToken);
                String verifyAuthToken = accessTokenMap.get(accessToken);
                if (StringUtils.isNoneBlank(verifyAuthToken)){
                    headers.put("verifyAuthToken",verifyAuthToken);
                } else {
                    headers.remove("verifyAuthToken");
                }
                String mallId = redisReader.listPop("pdd-mall-id-queue_liat");
                if (StringUtils.isBlank(mallId)){
                    try {
                        Thread.sleep(3000);
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }
                    continue;
                }

                try {
                    downloadBabyMilk("23082",mallId,downloader,httpConfig,headers,fileOutputStream,redisReader);
                    downloadBabyMilk("23994",mallId,downloader,httpConfig,headers,fileOutputStream,redisReader);

                } catch (Exception e) {
                    e.printStackTrace();
                }

                int index = accessTokens.indexOf(accessToken);
                index++;
                if (index > accessTokens.size()-1){
                    index = 0;
                }
                accessToken = accessTokens.get(index);
            }

        }

        private void downloadBabyMilk(String category_id,String shop,
                                      HttpClientDownloader downloader, HttpConfig httpConfig,
                                      Map<String, String> headers,FileOutputStream fileOutputStream,
                                      RedisReader redisReader) throws InterruptedException, IOException {
            String url = String.format(urlFormat, category_id, shop, 1);
            String goodIdsQueue = "pdd-html-shop-goods-id-queue_set";
            String cateGoodIdsQueue = "pdd-html-"+category_id+"-goods-id-queue_set";
            HttpRequest httpRequest = new HttpRequest();
            httpRequest.setHeaders(headers);
            HttpPage httpPage;
            httpRequest.setUrl(url);
            Json json = null;
            try {
                httpPage = downloader.download(httpRequest, httpConfig);
                json = new Json(httpPage.getRawText());
                List<String> all = json.jsonPath($_type + ".goods_list").all();
            } catch (Exception e) {
                boolean flag = true;
                while (flag){
                    try {
                        Random rand = new Random();
                        long sleepTime = (long)((rand.nextFloat()+1.5)*2000L);
                        Thread.sleep(sleepTime);

                        httpPage = downloader.download(httpRequest, httpConfig);
                        json = new Json(httpPage.getRawText());
                        List<String> all = json.jsonPath($_type + ".goods_list").all();
                        flag =false;
                    } catch (Exception ex) {
                        ex.printStackTrace();
                    }
                }
            }

            while (true) {
                Random rand = new Random();
                long sleepTime = (long)((rand.nextFloat()+1.5)*2000L);
                Thread.sleep(sleepTime);

                List<String> all = json.jsonPath($_type + ".goods_list").all();
                if (all == null ||all.size() < 1){
                    break;
                }
                List<String> results = new ArrayList<>();
                for (String s : all) {
                    try {
                        Json goodJson = new Json(s);
                        String goods_id = goodJson.jsonPath("$.goods_id").get();
                        if (redisReader.sismember(goodIdsQueue,goods_id)){
                            System.out.println("重复id："+goods_id);
                        } else {
                            Map objectMap = JSON.parseObject(s, Map.class);
                            objectMap.put("mall_id",shop);
                            results.add(JSON.toJSONString(objectMap));
                            redisReader.sadd(goodIdsQueue,goods_id);
                            redisReader.sadd(cateGoodIdsQueue,goods_id);
                        }

                    } catch (Exception e) {
                        logger.error("wash detail {}",s);
                    }
                }

                IOUtils.writeLines(results,"\r\n",fileOutputStream,"utf-8");

                int page_no = Integer.valueOf(json.jsonPath($_type + ".page_no").get());

                url = String.format(urlFormat, category_id, shop, ++page_no);
                httpRequest.setUrl(url);
                try {
                    httpPage = downloader.download(httpRequest, httpConfig);
                    json = new Json(httpPage.getRawText());
                } catch (Exception e) {
                    boolean flag = true;
                    while (flag){
                        try {
                            sleepTime = (long)((rand.nextFloat()+1.5)*2000L);
                            Thread.sleep(sleepTime);

                            httpPage = downloader.download(httpRequest, httpConfig);
                            json = new Json(httpPage.getRawText());
                            all = json.jsonPath($_type + ".goods_list").all();
                            flag =false;
                        } catch (Exception ex) {
                            ex.printStackTrace();
                        }
                    }
                }
            }
        }
    }

    static class TestProductDetailRun implements Runnable {

        HttpClientDownloader downloader;
        HttpConfig httpConfig;

        public TestProductDetailRun(HttpClientDownloader downloader, HttpConfig httpConfig) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
        }


        @Override
        public void run() {
            String produceDetailUrl = "https://api.pinduoduo.com/api/oak/integration/render?pdduid=3848719808912";

            Map<String,String> headers = new HashMap<>();
            headers.put("X-PDD-QUERIES","width=1080&height=1920&net=1&brand=Xiaomi&model=MI+5s&osv=8.0.0&appv=5.60.0&pl=2");
            headers.put("ETag","jVYb8J44");
            headers.put("AccessToken","36W7PJGKRND6DOLFHUG6CJWUSKMPMMK3WUCNVJD7HQCGWPJL4GKA111903b");
            headers.put("Referer","Android");
            headers.put("Content-Type","application/json;charset=UTF-8");
            headers.put("p-mediainfo","player=2.0.3&rtc=3.0.0");
            headers.put("p-appname","pinduoduo");
            headers.put("x-pdd-info","bold_free%3Dfalse%26bold_product%3D");
            headers.put("p-proc","main");
            headers.put("User-Agent","android Mozilla/5.0 (Linux; Android 8.0.0; MI 5s Build/OPR1.170623.032; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.99 Mobile Safari/537.36 phh_android_version/5.59.0 phh_android_build/8580855ea13a70c6bd60b68f6214008b255cc9af phh_android_channel/yyb pversion/0");
            headers.put("PDD-CONFIG","V4:001.055900");
            headers.put("multi-set","0,0");
            headers.put("Host","api.pinduoduo.com");
            headers.put("Connection","Keep-Alive");
            headers.put("Accept-Encoding","gzip");

            FileOutputStream productOutputStream = null;
            FileOutputStream productSkuOutputStream = null;
            try {
                productOutputStream = new FileOutputStream("D:\\chance\\data\\pdd\\pdd_product.json", true);
                productSkuOutputStream = new FileOutputStream("D:\\chance\\data\\pdd\\pdd_product_sku.json", true);
            } catch (FileNotFoundException e) {
                e.printStackTrace();
            }

            HttpRequest httpRequest = new HttpRequest();
            httpRequest.setHeaders(headers);
            HttpPage httpPage;

            String goodIdsQueue = "pdd-goods-id-queue_list";
            RedisReader redisReader = new RedisReader("192.168.1.215", 6379, 2);
            String goodsId = redisReader.listPop(goodIdsQueue);
            while (StringUtils.isNotBlank(goodsId)){
                try {
                    Random rand = new Random();
                    long sleepTime = (long)((rand.nextFloat()+1.5)*2000L);
                    Thread.sleep(sleepTime);

                    httpRequest.setUrl(produceDetailUrl);
                    httpRequest.setRequestBody(HttpRequestBody.json("{\"goods_id\":\""+goodsId+"\"}","utf-8"));
                    httpRequest.setMethod("POST");
                    httpPage = downloader.download(httpRequest, httpConfig);
                    Json json = new Json(httpPage.getRawText());
                    List<String> skus = json.jsonPath("$.sku").all();
                    IOUtils.writeLines(skus,"\r\n",productSkuOutputStream,"utf-8");
                    IOUtils.writeLines(Arrays.asList(httpPage.getRawText()),"\r\n",productOutputStream,"utf-8");
                } catch (Exception e) {
                    e.printStackTrace();
                    boolean flag = true;
                    while (flag){
                        try {
                            Random rand = new Random();
                            long sleepTime = (long)((rand.nextFloat()+1.5)*2000L);
                            Thread.sleep(sleepTime);

                            httpRequest.setUrl(produceDetailUrl);
                            httpRequest.setRequestBody(HttpRequestBody.json("{\"goods_id\":\""+goodsId+"\"}","utf-8"));
                            httpRequest.setMethod("POST");
                            httpPage = downloader.download(httpRequest, httpConfig);
                            Json json = new Json(httpPage.getRawText());
                            List<String> skus = json.jsonPath("$.sku").all();
                            IOUtils.writeLines(skus,"\r\n",productSkuOutputStream,"utf-8");
                            IOUtils.writeLines(Arrays.asList(httpPage.getRawText()),"\r\n",productOutputStream,"utf-8");
                            flag = false;
                        } catch (Exception e1) {
                            e1.printStackTrace();
                        }
                    }
                }
                goodsId = redisReader.listPop(goodIdsQueue);
            }

        }
    }

    static class TestCaterhamRun implements Runnable {

        HttpClientDownloader downloader;
        HttpConfig httpConfig;

        public TestCaterhamRun(HttpClientDownloader downloader, HttpConfig httpConfig) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
        }


        @Override
        public void run() {

            File file = new File("D:\\chance\\data\\pdd\\pdd.json");
            if (file.exists()){
                file.delete();
            }
            FileOutputStream fileOutputStream = null;
            try {
                fileOutputStream = new FileOutputStream("D:\\chance\\data\\pdd\\pdd.json", true);
            } catch (FileNotFoundException e) {
                e.printStackTrace();
            }

            String urlFormat = "https://api.pinduoduo.com/api/caterham/query/subfenlei_gyl_label?" +
                    "offset=%s" +
                    "&list_id=4939_2872a4ab02&count=20&opt_id=4939&req_list_action_type=6&page_sn=10028&page_id=catgoods.html&support_types=0_4_5&content_goods_num=4&size=20&opt_type=3&engine_version=2.0&req_action_type=6&pdduid=1720023679323";
//            String urlFormat = "https://api.pinduoduo.com/api/caterham/query/subfenlei_gyl_label?offset=%S&opt_type=3";
            Map<String,String> headers = new HashMap<>();
            headers.put("X-PDD-QUERIES","width=1080&height=1920&net=1&brand=Xiaomi&model=MI+5s&osv=8.0.0&appv=5.59.0&pl=2");
            headers.put("ETag","wYmUAFKX");
            headers.put("AccessToken","2H7AL5SPNLTHX32ZWSGCYDS5EUORTU6I5PA3ABQLG2NJ5W3VVZRA1115bc2");
            headers.put("Referer","Android");
            headers.put("lat","FGKT7DZVN4SNDBUEET363KUA5UDWJGUHUN2T55SKMU45VIPY4AKA1115bc2");
            headers.put("Content-Type","application/json;charset=UTF-8");
            headers.put("p-mediainfo","player=2.0.3&rtc=3.0.0");
            headers.put("VerifyAuthToken","0dZi1ox4ejaGy6WE1nAzLQ0844ccadbc94f4f96");
            headers.put("p-appname","pinduoduo");
            headers.put("x-pdd-info","bold_free%3Dfalse%26bold_product%3D");
            headers.put("p-proc","main");
//            headers.put("x-b3-ptracer","hctrueb4e0cbca15384f71be4b4350b0");
            headers.put("User-Agent","android Mozilla/5.0 (Linux; Android 8.0.0; MI 5s Build/OPR1.170623.032; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.99 Mobile Safari/537.36  phh_android_version/5.59.0 phh_android_build/6b78860f01997cbf5a34356e75a46680ad868719_pdd_patch phh_android_channel/yyb pversion/2787");
            headers.put("PDD-CONFIG","V4:001.055900");
            headers.put("multi-set","1,0");
            headers.put("Host","api.pinduoduo.com");
            headers.put("Connection","Keep-Alive");
            headers.put("Accept-Encoding","gzip");
            headers.put("Cookie","acid=62a0e59d4f9054342267397f7adc185f; JSESSIONID=1358F13D4496356D0A7EA18AF1768D13; api_uid=CkwyoWCHglw2cgBTPJc/Ag==");

            HttpRequest httpRequest = new HttpRequest();
            httpRequest.setHeaders(headers);
            HttpPage httpPage;
            Set<String> goodIds = new HashSet<>();
            try {
                String url = String.format(urlFormat, 0);
                httpRequest.setUrl(url);
                httpPage = downloader.download(httpRequest, httpConfig);
                Json json = new Json(httpPage.getRawText());
                String hasMore = json.jsonPath("$.has_more").get();
                int offset = 0;

                while ("true".equals(hasMore)){
                    try {
                        Random rand = new Random();
                        long sleepTime = (long)((rand.nextFloat()+1.5)*2000L);
                        Thread.sleep(sleepTime);

                        offset += 20;
                        List<String> all = json.jsonPath($_type + ".goods_list").all();
                        List<String> goods = new ArrayList<>();
                        for (String s : all) {
                            Json goodJson = new Json(s);
                            String goods_id = goodJson.jsonPath("$.goods_id").get();
                            String thumb_url = goodJson.jsonPath("$.thumb_url").get();
//                            imageDownload(thumb_url,downloader,httpConfig);
                            if (goodIds.contains(goods_id)){
                                System.out.println("重复id："+goods_id);
                            }
                            goodIds.add(goods_id);
                            downloadGoodDetail(downloader,httpConfig,goods_id,goods);
                        }
                        IOUtils.writeLines(goods,"\r\n",fileOutputStream,"utf-8");

                        url = String.format(urlFormat, offset);
                        httpRequest.setUrl(url);
                        httpPage = downloader.download(httpRequest, httpConfig);
                        json = new Json(httpPage.getRawText());
                        hasMore = json.jsonPath("$.has_more").get();
                    } catch (Exception e) {
                        e.printStackTrace();
                        hasMore = null;
                        while (StringUtils.isBlank(hasMore)){
                            try {
                                Random rand = new Random();
                                long sleepTime = (long)((rand.nextFloat()+1.5)*2000L);
                                Thread.sleep(sleepTime);
                                url = String.format(urlFormat, offset);
                                httpRequest.setUrl(url);
                                httpPage = downloader.download(httpRequest, httpConfig);
                                json = new Json(httpPage.getRawText());
                                hasMore = json.jsonPath("$.has_more").get();
                            } catch (Exception e1) {

                            }
                        }
                    }
                }
            } catch (Exception e) {
                e.printStackTrace();
            }

        }
    }

    static class TestBabyMilkRun implements Runnable {

        HttpClientDownloader downloader;
        HttpConfig httpConfig;

        public TestBabyMilkRun(HttpClientDownloader downloader, HttpConfig httpConfig) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
        }


        @Override
        public void run() {

            File file = new File("D:\\chance\\data\\pdd\\pdd.json");
            if (file.exists()){
                file.delete();
            }
            FileOutputStream fileOutputStream = null;
            try {
                fileOutputStream = new FileOutputStream("D:\\chance\\data\\pdd\\pdd.json", true);
            } catch (FileNotFoundException e) {
                e.printStackTrace();
            }

            String urlFormat = "https://api.pinduoduo.com/api/search/opt/9972/groups?" +
                    "sort_type=default&page_sn=23699&size=50" +
                    "&offset=%s" +
                    "&list_id=9972_28e6f112bd&opt_source=search_opt_goods&white_ground_pic_goods_id=191354089752&opt_type=3&back_search=false&pdduid=1720023679323";
            String otherUrlFormat = "https://api.pinduoduo.com/api/search/opt/9972/groups?" +
                    "filter=property_tag,-8,&sort_type=default&page_sn=23699&size=50" +
                    "&offset=%s" +
                    "&list_id=9972_89d6b1fcb7&opt_source=search_opt_goods&white_ground_pic_goods_id=191354089752&opt_type=3" +
                    "&flip=%s" +
                    "&back_search=false&pdduid=1720023679323";
            Map<String,String> headers = new HashMap<>();
            headers.put("X-PDD-QUERIES","width=1080&height=1920&net=1&brand=Xiaomi&model=MI+5s&osv=8.0.0&appv=5.59.0&pl=2");
            headers.put("ETag","wYmUAFKX");
            headers.put("AccessToken","2H7AL5SPNLTHX32ZWSGCYDS5EUORTU6I5PA3ABQLG2NJ5W3VVZRA1115bc2");
            headers.put("Referer","Android");
            headers.put("lat","FGKT7DZVN4SNDBUEET363KUA5UDWJGUHUN2T55SKMU45VIPY4AKA1115bc2");
            headers.put("Content-Type","application/json;charset=UTF-8");
            headers.put("p-mediainfo","player=2.0.3&rtc=3.0.0");
            headers.put("VerifyAuthToken","0dZi1ox4ejaGy6WE1nAzLQ0844ccadbc94f4f96");
            headers.put("p-appname","pinduoduo");
            headers.put("x-pdd-info","bold_free%3Dfalse%26bold_product%3D");
            headers.put("p-proc","main");
//            headers.put("x-b3-ptracer","hctrueb4e0cbca15384f71be4b4350b0");
            headers.put("User-Agent","android Mozilla/5.0 (Linux; Android 8.0.0; MI 5s Build/OPR1.170623.032; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.99 Mobile Safari/537.36  phh_android_version/5.59.0 phh_android_build/6b78860f01997cbf5a34356e75a46680ad868719_pdd_patch phh_android_channel/yyb pversion/2787");
            headers.put("PDD-CONFIG","V4:001.055900");
            headers.put("multi-set","0,0");
            headers.put("Host","api.pinduoduo.com");
            headers.put("Connection","Keep-Alive");
            headers.put("Accept-Encoding","gzip");
            headers.put("Cookie","acid=0400b4c335f41ac6b0c1ae581efb40aa; JSESSIONID=26A6AFC04E7E5BEB26D2BA6972879B4F; api_uid=CkwyoWCHglw2cgBTPJc/Ag==");

            HttpRequest httpRequest = new HttpRequest();
            httpRequest.setHeaders(headers);
            HttpPage httpPage;
            Set<String> goodIds = new HashSet<>();
            try {
                String url = String.format(urlFormat, 0);
                httpRequest.setUrl(url);
                httpPage = downloader.download(httpRequest, httpConfig);
                Json json = new Json(httpPage.getRawText());
                String flip = json.jsonPath("$.flip").get();
                int offset = Integer.valueOf(json.jsonPath("$.p_search.offset").get());

                while (StringUtils.isNotBlank(flip)){
                    try {
                        Random rand = new Random();
                        long sleepTime = (long)((rand.nextFloat()+1.5)*2000L);
                        Thread.sleep(sleepTime);

                        offset += 50;
                        List<String> all = json.jsonPath($_type + ".goods_list").all();
                        List<String> goods = new ArrayList<>();
                        for (String s : all) {
                            Json goodJson = new Json(s);
                            String goods_id = goodJson.jsonPath("$.goods_id").get();
                            String thumb_url = goodJson.jsonPath("$.thumb_url").get();
//                            imageDownload(thumb_url,downloader,httpConfig);
                            if (goodIds.contains(goods_id)){
                                System.out.println("重复id："+goods_id);
                            }
                            goodIds.add(goods_id);
//                            downloadGoodDetail(downloader,httpConfig,goods_id,goods);
                        }
                        IOUtils.writeLines(all,"\r\n",fileOutputStream,"utf-8");

                        url = String.format(urlFormat, offset, URLEncoder.encode(flip,"utf-8"));
                        httpRequest.setUrl(url);
                        httpPage = downloader.download(httpRequest, httpConfig);
                        json = new Json(httpPage.getRawText());
                        flip = json.jsonPath("$.flip").get();
                        offset = Integer.valueOf(json.jsonPath("$.p_search.offset").get());
                    } catch (Exception e) {
                        e.printStackTrace();
                        flip = null;
                        while (StringUtils.isBlank(flip)){
                            try {
                                Random rand = new Random();
                                long sleepTime = (long)((rand.nextFloat()+1.5)*2000L);
                                Thread.sleep(sleepTime);
                                url = String.format(urlFormat, offset);
                                httpRequest.setUrl(url);
                                httpPage = downloader.download(httpRequest, httpConfig);
                                json = new Json(httpPage.getRawText());
                                flip = json.jsonPath("$.flip").get();
                                offset = Integer.valueOf(json.jsonPath("$.p_search.offset").get());
                            } catch (Exception e1) {

                            }
                        }
                    }
                }
            } catch (Exception e) {
                e.printStackTrace();
            }

        }
    }

    private static void downloadGoodDetail(HttpClientDownloader downloader, HttpConfig httpConfig,
                                           String goodsId,List<String> goods){
        HttpRequest httpRequest = new HttpRequest();
        String url = "https://api.pinduoduo.com/api/oak/integration/render?pdduid=1720023679323";
        String urlParamForat= "{\"address_list\":[],\"goods_id\":\"%s\",\"page_from\":\"24\",\"page_version\":\"7\"," +
                "\"client_time\":\"%s\",\"refer_page_sn\":\"10028\",\"refer_page_el_sn\":\"98978\",\"extend_map\":{}}";
        Map<String,String> headers = new HashMap<>();
        headers.put("X-PDD-QUERIES","width=1080&height=1920&net=1&brand=Xiaomi&model=MI+5s&osv=8.0.0&appv=5.59.0&pl=2");
        headers.put("ETag","wYmUAFKX");
        headers.put("AccessToken","2H7AL5SPNLTHX32ZWSGCYDS5EUORTU6I5PA3ABQLG2NJ5W3VVZRA1115bc2");
        headers.put("Referer","Android");
        headers.put("lat","FGKT7DZVN4SNDBUEET363KUA5UDWJGUHUN2T55SKMU45VIPY4AKA1115bc2");
        headers.put("Content-Type","application/json;charset=UTF-8");
        headers.put("p-mediainfo","player=2.0.3&rtc=3.0.0");
        headers.put("p-appname","pinduoduo");
        headers.put("x-pdd-info","bold_free%3Dfalse%26bold_product%3D");
        headers.put("p-proc","main");
//            headers.put("x-b3-ptracer","hctrueb4e0cbca15384f71be4b4350b0");
        headers.put("User-Agent","android Mozilla/5.0 (Linux; Android 8.0.0; MI 5s Build/OPR1.170623.032; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.99 Mobile Safari/537.36  phh_android_version/5.59.0 phh_android_build/6b78860f01997cbf5a34356e75a46680ad868719_pdd_patch phh_android_channel/yyb pversion/2787");
        headers.put("PDD-CONFIG","V4:001.055900");
        headers.put("multi-set","0,0");
        headers.put("Host","api.pinduoduo.com");
        headers.put("Connection","Keep-Alive");
        headers.put("Accept-Encoding","gzip");
        headers.put("Cookie","acid=62a0e59d4f9054342267397f7adc185f; JSESSIONID=1358F13D4496356D0A7EA18AF1768D13; api_uid=CkwyoWCHglw2cgBTPJc/Ag==");

        HttpRequestBody requestBody = HttpRequestBody.json(String.format(urlParamForat, goodsId, System.currentTimeMillis()), "utf-8");
        httpRequest.setUrl(url);
        httpRequest.setHeaders(headers);
        httpRequest.setMethod("POST");
        httpRequest.setRequestBody(requestBody);
        HttpPage httpPage = null;
        try {
            httpPage = downloader.download(httpRequest, httpConfig);
            Json json = new Json(httpPage.getRawText());
            json.jsonPath("$.goods.goods_id").get();
        } catch (Exception e) {
            e.printStackTrace();
            goodsId = null;
            while (StringUtils.isBlank(goodsId)){
                try {

                    httpPage = downloader.download(httpRequest, httpConfig);
                    Json json = new Json(httpPage.getRawText());
                    goodsId = json.jsonPath("$.goods.goods_id").get();

                } catch (Exception e1) {

                }
            }
        }
        goods.add(httpPage.getRawText());

    }

    static class TestCluCaterhamRun implements Runnable {

        HttpClientDownloader downloader;
        HttpConfig httpConfig;

        public TestCluCaterhamRun(HttpClientDownloader downloader, HttpConfig httpConfig) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
        }


        @Override
        public void run() {

            File file = new File("D:\\chance\\data\\pdd\\pdd.json");
            if (file.exists()){
                file.delete();
            }
            FileOutputStream fileOutputStream = null;
            try {
                fileOutputStream = new FileOutputStream("D:\\chance\\data\\pdd\\pdd.json", true);
            } catch (FileNotFoundException e) {
                e.printStackTrace();
            }

            String urlFormat = "https://api.pinduoduo.com/api/caterham/query/subfenlei_gyl_label?" +
                    "offset=%s" +
                    "&list_id=4936_ebab952e96&count=20&opt_id=4936&req_list_action_type=6&page_sn=10028&page_id=catgoods.html&support_types=0_4_5&content_goods_num=4&size=20&opt_type=3&engine_version=2.0&req_action_type=10&pdduid=";
//            String urlFormat = "https://api.pinduoduo.com/api/caterham/query/subfenlei_gyl_label?offset=%S&opt_type=3";
            Map<String,String> headers = new HashMap<>();
            headers.put("X-PDD-QUERIES","width=1080&height=1920&net=1&brand=Xiaomi&model=MI+5s&osv=8.0.0&appv=5.59.0&pl=2");
            headers.put("ETag","wYmUAFKX");
            headers.put("Referer","Android");
            headers.put("Content-Type","application/json;charset=UTF-8");
            headers.put("p-mediainfo","player=2.0.3&rtc=3.0.0");
            headers.put("p-appname","pinduoduo");
            headers.put("x-pdd-info","bold_free%3Dfalse%26bold_product%3D");
            headers.put("p-proc","main");
//            headers.put("x-b3-ptracer","hctrueb4e0cbca15384f71be4b4350b0");
            headers.put("User-Agent","android Mozilla/5.0 (Linux; Android 8.0.0; MI 5s Build/OPR1.170623.032; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/71.0.3578.99 Mobile Safari/537.36  phh_android_version/5.59.0 phh_android_build/6b78860f01997cbf5a34356e75a46680ad868719_pdd_patch phh_android_channel/yyb pversion/2787");
            headers.put("PDD-CONFIG","V4:001.055900");
            headers.put("multi-set","1,0");
            headers.put("Host","api.pinduoduo.com");
            headers.put("Connection","Keep-Alive");
            headers.put("Accept-Encoding","gzip");
            headers.put("Cookie","acid=fbdd57d3de1fad139b9c034b7032846b; JSESSIONID=0F252D89F697D9E61C6A8FAEF39DBB9A; api_uid=CkwyoWCHglw2cgBTPJc/Ag==");

            HttpRequest httpRequest = new HttpRequest();
            httpRequest.setHeaders(headers);
            HttpPage httpPage;
            Set<String> goodIds = new HashSet<>();
            try {
                String url = String.format(urlFormat, 0);
                httpRequest.setUrl(url);
                httpPage = downloader.download(httpRequest, httpConfig);
                Json json = new Json(httpPage.getRawText());
                String hasMore = json.jsonPath("$.has_more").get();
                int offset = 0;

                while ("true".equals(hasMore)){
                    try {
                        Random rand = new Random();
                        long sleepTime = (long)((rand.nextFloat()+1.5)*2000L);
                        Thread.sleep(sleepTime);

                        offset += 20;
                        List<String> all = json.jsonPath($_type + ".goods_list").all();
                        for (String s : all) {
                            Json goodJson = new Json(s);
                            String goods_id = goodJson.jsonPath("$.goods_id").get();
                            String thumb_url = goodJson.jsonPath("$.thumb_url").get();
                            imageDownload(thumb_url,downloader,httpConfig);
                            if (goodIds.contains(goods_id)){
                                System.out.println("重复id："+goods_id);
                            }
                            goodIds.add(goods_id);
                        }
                        IOUtils.writeLines(all,"\r\n",fileOutputStream,"utf-8");

                        url = String.format(urlFormat, offset);
                        httpRequest.setUrl(url);
                        httpPage = downloader.download(httpRequest, httpConfig);
                        json = new Json(httpPage.getRawText());
                        hasMore = json.jsonPath("$.has_more").get();
                    } catch (Exception e) {
                        switchIp(downloader,httpConfig);
                        e.printStackTrace();
                        hasMore = null;
                        while (StringUtils.isBlank(hasMore)){
                            try {
                                Random rand = new Random();
                                long sleepTime = (long)((rand.nextFloat()+1.5)*2000L);
                                Thread.sleep(sleepTime);
                                url = String.format(urlFormat, offset);
                                httpRequest.setUrl(url);
                                httpPage = downloader.download(httpRequest, httpConfig);
                                json = new Json(httpPage.getRawText());
                                hasMore = json.jsonPath("$.has_more").get();
                            } catch (Exception e1) {
                                switchIp(downloader,httpConfig);
                            }
                        }
                    }
                }
            } catch (Exception e) {
                e.printStackTrace();
            }

        }
    }

    private static void switchIp(HttpClientDownloader downloader, HttpConfig httpConfig){
        HttpRequest currentIPRequest = new HttpRequest();
        currentIPRequest.setUrl("http://proxy.abuyun.com/current-ip");
        HttpPage httpPage = downloader.download(currentIPRequest, httpConfig);
        logger.info("current ip {}",httpPage.getRawText());
        switchIp(downloader,httpConfig,httpPage.getRawText().split(",")[0]);
    }

    private static void switchIp(HttpClientDownloader downloader, HttpConfig httpConfig,String currentIp){


        HttpRequest httpRequest = new HttpRequest();
        httpRequest.setUrl("http://proxy.abuyun.com/switch-ip");

        HttpPage httpPage = downloader.download(httpRequest, httpConfig);
        String switchIP = httpPage.getRawText().split(",")[0];

        while (ips.contains(switchIP)){
            try {
                Thread.sleep(700);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
            httpPage = downloader.download(httpRequest,httpConfig);
            switchIP = httpPage.getRawText().split(",")[0];
        }
        ips.add(switchIP);
        logger.info("switch ip: " + httpPage.getRawText());

    }

    private static void imageDownload(String imageUrl,HttpClientDownloader downloader, HttpConfig httpConfig){
        if (StringUtils.isNotBlank(imageUrl)){
            HttpRequest httpRequest = new HttpRequest();
            httpRequest.setUrl(imageUrl);
            httpConfig.setResponseTextGenerateHtml(false);
            downloader.download(httpRequest, httpConfig);
        }
    }

    static class TestCaterhamHtmlRun implements Runnable {

        HttpClientDownloader downloader;
        HttpConfig httpConfig;

        public TestCaterhamHtmlRun(HttpClientDownloader downloader, HttpConfig httpConfig) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
        }


        @Override
        public void run() {
            String urlFormat = "https://api.pinduoduo.com/api/caterham/query/subfenlei_gyl_label?" +
                    "offset=%s" +
                    "&list_id=4936_2d186cb24c&count=20&opt_id=4936&req_list_action_type=0&page_sn=10028&page_id=catgoods.html&support_types=0_4_5&content_goods_num=4&size=20&opt_type=3&engine_version=2.0&req_action_type=10&pdduid=1720023679323";
//            String urlFormat = "http://yangkeduo.com/proxy/api/api/caterham/query/subfenlei_gyl_label?pdduid=6968271439966&page_sn=10028" +
//                    "&support_types=0_4&opt_id=4936&opt_type=3&offset=%s&count=20&list_id=vu2uyhd48t_4936";
            Map<String,String> headers = new HashMap<>();
            headers.put("Pragma","no-cache");
            headers.put("Cache-Control","no-cache");
            headers.put("Cookie ","acid=fbdd57d3de1fad139b9c034b7032846b; JSESSIONID=B38C23B67AA95845AC89D20B3D91FE4D; api_uid=CkwyoWCHglw2cgBTPJc/Ag==");
//            headers.put("Cache-Control","no-cache");
            headers.put("AccessToken","MROOKOIDMZVXQMUDZYN5AY3EP6TFTAGL5KDFYWTL4Q6YNTYGMIEQ1115bc2");
            headers.put("Accept","application/json, text/plain, */*");
            headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36");
            headers.put("Referer","http://yangkeduo.com/catgoods.html?refer_page_name=index&opt_id=4935&opt_name=%E5%A5%B6%E7%B2%89%E8%BE%85%E9%A3%9F&opt_type=2&goods_id=1488049922&refer_page_id=10002_1619597076665_dmm7sayav9&refer_page_sn=10002");
            File file = new File("D:\\chance\\data\\pdd\\pdd.json");
            if (file.exists()){
                file.delete();
            }
            FileOutputStream fileOutputStream = null;
            try {
                fileOutputStream = new FileOutputStream("D:\\chance\\data\\pdd\\pdd.json", true);
            } catch (FileNotFoundException e) {
                e.printStackTrace();
            }

            List<String> keyWords = genKeywords();
            HttpRequest httpRequest = new HttpRequest();
            httpRequest.setHeaders(headers);
            HttpPage httpPage;
            try {
                String url = String.format(urlFormat, 0);
                httpRequest.setUrl(url);
                httpPage = downloader.download(httpRequest, httpConfig);
                Json json = new Json(httpPage.getRawText());
                String hasMore = json.jsonPath("$.has_more").get();
                int offset = 0;

                while ("true".equals(hasMore)){
                    try {
                        offset += 20;
                        List<String> all = json.jsonPath($_type + ".goods_list").all();
                        if (all !=null && all.size() >0){
                            for (String s : all) {
                                System.out.println(s);
                            }
                            IOUtils.writeLines(all,"\r\n",fileOutputStream,"utf-8");
                        }

                        url = String.format(urlFormat, offset);
                        httpRequest.setUrl(url);
                        httpPage = downloader.download(httpRequest, httpConfig);
                        json = new Json(httpPage.getRawText());
                        hasMore = json.jsonPath("$.has_more").get();
                    } catch (Exception e) {
                        e.printStackTrace();
                        hasMore = null;
                        while (StringUtils.isBlank(hasMore)){
                            try {
                                url = String.format(urlFormat, offset);
                                httpRequest.setUrl(url);
                                httpPage = downloader.download(httpRequest, httpConfig);
                                json = new Json(httpPage.getRawText());
                                hasMore = json.jsonPath("$.has_more").get();
                            } catch (Exception e1) {

                            }
                        }
                    }
                }

                System.out.println("--end--");
            } catch (Exception e) {
                e.printStackTrace();
            }

        }
    }

    static class TestBabyMilkHtmlRun implements Runnable {

        HttpClientDownloader downloader;
        HttpConfig httpConfig;

        public TestBabyMilkHtmlRun(HttpClientDownloader downloader, HttpConfig httpConfig) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
        }


        @Override
        public void run() {
            String urlFormat = "http://yangkeduo.com/proxy/api/api/search/opt/9972/groups?" +
                    "pdduid=6968271439966&source=search&opt_type=3&white_ground_pic_goods_id=136708874942&flip=&" +
                    "offset=%s" +
                    "&count=20&list_id=BxHNSHOOdI&sort=default&filter=&opt_source=search_opt_goods";
            String otherUrlFormat = "http://yangkeduo.com/proxy/api/api/search/opt/9972/groups?" +
                    "pdduid=6968271439966&source=search&opt_type=3&white_ground_pic_goods_id=136708874942" +
                    "&flip=%s" +
                    "&offset=%s" +
                    "&count=20&list_id=BxHNSHOOdI&sort=default&filter=&opt_source=search_opt_goods";
            Map<String,String> headers = new HashMap<>();
            headers.put("Pragma","no-cache");
            headers.put("Cache-Control","no-cache");
            headers.put("Cookie","api_uid=CkrNdmCJFm8PrgBXF3E1Ag==; _nano_fp=XpEalpdbn5UanpX8l9_w5OQ6E6JTOLRwUP6yuhn9; PDDAccessToken=3ZWZ2E5MSEC32Q2DPA52TSZ5RZNKJV3JMGWXXESC4DT734EBRA3A1105ea2; pdd_user_id=6968271439966; pdd_user_uin=5VUAIDCCJOICP2OLV6XEVVDBVM_GEXDA; ua=Mozilla%2F5.0%20(Windows%20NT%2010.0%3B%20Win64%3B%20x64)%20AppleWebKit%2F537.36%20(KHTML%2C%20like%20Gecko)%20Chrome%2F90.0.4430.93%20Safari%2F537.36; webp=1; pdd_vds=gaLLNIOEnObLLnNGPaEPbnONaPyPQaGtINaOaGENiyGNyGyGPynQnbLiEmLn; JSESSIONID=39E006324F64B06381893FE0E8170CCD");
            headers.put("Connection","keep-alive");
            headers.put("AccessToken","3ZWZ2E5MSEC32Q2DPA52TSZ5RZNKJV3JMGWXXESC4DT734EBRA3A1105ea2");
            headers.put("Accept","application/json, text/plain, */*");
            headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36");
            headers.put("Referer","http://yangkeduo.com/search_catgoods.html?opt_id=9972&opt1_id=9907&opt2_id=999999&opt_g=1&opt_type=3&opt_name=%E5%A9%B4%E5%84%BF%E5%A5%B6%E7%B2%89&_x_link_id=04871431-605c-4203-8b37-989d4127b57f&_x_goods_id=136708874942&refer_page_name=search&refer_page_id=10031_1619683649385_vkujxhmd18&refer_page_sn=10031");
            File file = new File("D:\\chance\\data\\pdd\\pdd.json");
            if (file.exists()){
                file.delete();
            }
            FileOutputStream fileOutputStream = null;
            try {
                fileOutputStream = new FileOutputStream("D:\\chance\\data\\pdd\\pdd.json", true);
            } catch (FileNotFoundException e) {
                e.printStackTrace();
            }

            HttpRequest httpRequest = new HttpRequest();
            httpRequest.setHeaders(headers);
            HttpPage httpPage;
            Set<String> goodIds = new HashSet<>();
            try {
                String url = String.format(urlFormat, 0);
                httpRequest.setUrl(url);
                httpPage = downloader.download(httpRequest, httpConfig);
                Json json = new Json(httpPage.getRawText());
                String flip = json.jsonPath("$.flip").get();
                int offset = Integer.valueOf(json.jsonPath("$.p_search.offset").get());
                String lastFlip = flip;
                while (StringUtils.isNotBlank(flip)){
                    try {
                        Random rand = new Random();
                        long sleepTime = (long)((rand.nextFloat()+1.5)*2000L);
                        Thread.sleep(sleepTime);

                        offset += 20;
                        List<String> all = json.jsonPath($_type + ".goods_list").all();
                        List<String> goods = new ArrayList<>();
                        for (String s : all) {
                            Json goodJson = new Json(s);
                            String goods_id = goodJson.jsonPath("$.goods_id").get();
                            String thumb_url = goodJson.jsonPath("$.thumb_url").get();
//                            imageDownload(thumb_url,downloader,httpConfig);
                            if (goodIds.contains(goods_id)){
                                System.out.println("重复id："+goods_id);
                            }
                            goodIds.add(goods_id);
//                            downloadGoodDetail(downloader,httpConfig,goods_id,goods);
                        }
                        IOUtils.writeLines(all,"\r\n",fileOutputStream,"utf-8");

                        url = String.format(otherUrlFormat, URLEncoder.encode(flip,"utf-8"),offset);
                        httpRequest.setUrl(url);
                        httpPage = downloader.download(httpRequest, httpConfig);
                        json = new Json(httpPage.getRawText());
                        flip = json.jsonPath("$.flip").get();
                        offset = Integer.valueOf(json.jsonPath("$.p_search.offset").get());
                        lastFlip = flip;
                    } catch (Exception e) {
                        e.printStackTrace();
                        flip = null;
                        while (StringUtils.isBlank(flip)){
                            try {
                                Random rand = new Random();
                                long sleepTime = (long)((rand.nextFloat()+1.5)*2000L);
                                Thread.sleep(sleepTime);
                                url = String.format(otherUrlFormat, URLEncoder.encode(lastFlip,"utf-8"),offset);
                                httpRequest.setUrl(url);
                                httpPage = downloader.download(httpRequest, httpConfig);
                                json = new Json(httpPage.getRawText());
                                flip = json.jsonPath("$.flip").get();
                                offset = Integer.valueOf(json.jsonPath("$.p_search.offset").get());
                                lastFlip = flip;
                            } catch (Exception e1) {

                            }
                        }
                    }
                }

                System.out.println("--end--");
            } catch (Exception e) {
                e.printStackTrace();
            }

        }
    }

}
